package com.zhang.spark_2.com.zhang.core.wc

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @title:
 * @author: zhang
 * @date: 2022/2/17 09:47 
 */
object WC_groupBy {

  def main(args: Array[String]): Unit = {
    //TODO 获取环境
    val conf = new SparkConf().setMaster("local").setAppName("WordCount")
    val sc = new SparkContext(conf);

    val rdd = sc.makeRDD(
      List(("a", 1), ("b", 2), ("a", 1), ("b", 2), ("a", 1), ("b", 2)), 2
    )

    // todo 使用groupBy实现wordCount
    val group: RDD[(String, Iterable[(String, Int)])] = rdd.groupBy(_._1)
    group.mapValues(
      data => {
        data.map(_._2).sum
      }
    )
      .collect().foreach(println)

    sc.stop()
  }

}
